# -*- coding: utf-8 -*-
from scrapy.selector import Selector
import scrapy
import json
from scrapy.loader import ItemLoader, Identity
from spiders.items import GateItem
import time


class GateSpider(scrapy.Spider):
    # 爬虫名称
    name = "gate"
    # 设置允许的域名
    allowed_domains = ["gate.io"]
    base_url = 'https://gate.io'
    # 设置开始爬去的页面
    start_urls = ["https://gate.io/articlelist/ann"]
    # 自定义配置 pipeline
    custom_settings = {
        # 默认是DEBUG，即打印DEBUG, INFO, WARNING, ERROR，所有LOG信息
        'LOG_LEVEL': 'DEBUG',
        # 默认管道配置
        'ITEM_PIPELINES': {
            'spiders.pipelines.GatePipeline.GatePipeline': 1,
        },
        # 默认headers请求头配置(解决爬取为英文问题)
        'DEFAULT_REQUEST_HEADERS': {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Encoding': 'gzip, deflate, br',
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0',
            'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        },
        # 减慢爬取速度为1s
        'DOWNLOAD_DELAY': 1,
        # 指定redis数据库的连接参数
        'REDIS_HOST': '127.0.0.1',
        # 端口
        'REDIS_PORT': 6379,
        # 指定 redis链接密码，和使用哪一个数据库
        'REDIS_PARAMS': {
            'password': 'Aa123456',
            'db': 1,
        },
    }
    # 设置抓取到的文章分类ID
    cate_id = 1

    def parse(self, response):
        sel = Selector(response)
        link = sel.xpath("//div[@id='lcontentnews']/div[@class='latnewslist']//a/@href").extract()
        for x in link:
            request = scrapy.Request(self.base_url + x, callback=self.parse_item)
            print(x)
            yield request

    def parse_item(self, response):
        sel = Selector(response)
        item = ItemLoader(item=GateItem(), response=response)
        item.add_xpath('title', "//div[@class='dtl-title']/h2/text()")
        click_string = sel.xpath("//div[@class='new-dtl-info']/text()").extract()[1]
        click = int(click_string.replace('阅读：', '').replace(' ', ''))
        item.add_value('click', click)
        item.add_xpath('content', "//div[@class='dtl-content']")
        item.add_xpath('desc', "//meta[@name='description']/@content")
        item.add_value('cate_id', self.cate_id)
        item.add_value('source', response.url)
        item.add_value('author', 'gate.io')
        item.add_value('tag', '')
        item.add_value('create_time', int(time.time()))
        return item.load_item()
